package com.test.chatrobot3.test;

import java.util.Arrays;
import java.util.List;

public class TextTokenizer {
    public static List<String> tokenize(String text) {
        // 按空格分词
        return Arrays.asList(text.split("\\s+"));
    }

    public static void main(String[] args) {
        String text = "hello world this is a test";
        List<String> tokens = tokenize(text);
        System.out.println(tokens);  // 输出: [hello, world, this, is, a, test]
    }
}